# Import the required packages.
# Standard Packages
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import math
# Data preparation.
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
# Image Processing
from skimage.color import rgb2gray
from skimage.filters import gaussian
# Trnsorflow and Keras
import tensorflow as tf
from tensorflow.keras.utils import to_categorical
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Conv2D, MaxPooling2D, GlobalMaxPooling2D, Flatten, Dense, Dropout, BatchNormalization
from tensorflow.keras import optimizers
# Performance Metrics
from sklearn.metrics import accuracy_score, classification_report, confusion_matrix
# Do some formatting of the page.
from IPython.core.interactiveshell import InteractiveShell
pd.options.display.float_format = '{:,.2f}'.format
from IPython.core.display import display, HTML
display(HTML("<style>.container { width:90% !important; }</style>"))
pd.set_option('display.max_columns', None)
pd.set_option('display.float_format', '{:0.4f}'.format)
# If running locally with GPU. Do this to prevent errors due to GPU memory limit.
physical_devices = tf.config.experimental.list_physical_devices('GPU')
if len(physical_devices) > 0:
tf.config.experimental.set_memory_growth(physical_devices[0], True)
# Import the data.
images = np.load("images.npy")
labels = pd.read_csv("labels.csv")
# View the shape of the images array.
images.shape
images[0, 0, 0]
# View the shape of the labels array.
labels.shape
labels.info()
labels.nunique()
# The labels are grouped by Label in the csv file, get the positions where each new group begins.
# This will allow us to view information from each group, in both the labels and images arrays, more easily.
def getClassPositions(classList):
classPositions = []
classPositions.append(0)
currentClass = classList[0]
for i in range(len(classList)):
if(currentClass != classList[i]):
classPositions.append(i)
currentClass = classList[i]
return classPositions
classPositions = getClassPositions(labels['Label'])
classPositions
# Display the labels.
labels.iloc[classPositions]
# Convert the labels dataframe to a list,it will be easier to handle.
labelsList = list(labels['Label'])
# Print the first image from every class.
def printImages(imageArray, labelArray, positions, colorMap=None, picSize=5):
''' Prints the images at "positions" in rows of 6 images '''
totalImages = len(positions)
imageCount = 0
rows = math.ceil(totalImages/6)
lastRowCols = totalImages%6
f, ax = plt.subplots(rows, 6, figsize=(25,(rows*picSize)))
if (rows>1):
for i in range(rows):
for j in range(6):
ax[i,j].imshow(imageArray[positions[imageCount]], colorMap)
ax[i,j].title.set_text(labelArray[positions[imageCount]])
imageCount = imageCount + 1
if(imageCount == (totalImages)):
return
else:
for i in range(6):
ax[i].imshow(imageArray[positions[imageCount]], colorMap)
ax[i].title.set_text(labelArray[positions[imageCount]])
imageCount = imageCount + 1
if(imageCount == (totalImages)):
return
printImages(images, labelsList, classPositions)
# If you want to see more images of each class.
# Print the first 5 images from every class.
'''
for i, pos in enumerate(classPositions):
f, ax = plt.subplots(1, 5, figsize=(25,4))
f.suptitle(labels['Label'][pos], fontsize=12)
for j in range(5):
ax[j].imshow(images[pos+j])
'''
# Scale the images from int of 0-255 to a float of 0-1.
images_rgb = images.copy()
images_rgb = images_rgb.astype('float32')
images_rgb = images_rgb/255
images_rgb[0,0,0]
# Perform Gaussian blurring on the images.
for i in range(images_rgb.shape[0]):
images_rgb[i] = gaussian(images_rgb[i], sigma=1.0, multichannel=True)
# Print the gaussian blurred first image from every class.
printImages(images_rgb, labelsList, classPositions)
# Sometimes CNNs only need grayscale images, so create grayscale images as well.
# Convert the image array to gray scale.
images_gs = rgb2gray(images)
images_gs.shape
# Print the first image from every class in grayscale.
printImages(images_gs, labelsList, classPositions, 'gray')
# The rgb2gray function converts the images from an int between 0-255 to a float between 0-1.
# Confirms this.
images_gs[0, 0, 0] # Show the first pixel value to confirm
# Perform Gaussian blurring on the grayscale images.
for i in range(images_gs.shape[0]):
images_gs[i] = gaussian(images_gs[i], sigma=1.0, multichannel=True)
# Confirm that the pixel values are still floating point and between 0 and 1.
images_gs[0, 0, 0] # Show the first pixel value to confirm
# Print the grayscale gaussian blurred first image from every class.
printImages(images_gs, labelsList, classPositions, 'gray')
# Data is still floating point with values between 0 and 1, so there is no need for normalization.
images_gs[0,0,0]
# Display the labels.
labels.iloc[classPositions]
# Make a series to store the class names as they are in the current order.
classNames = pd.Series(np.array(labels.iloc[classPositions]['Label']))
classNames
# Rename the classes with alphabet letters in the order which they appear.
# This is not neccessary, but it will keep the columns in the same order as which the labels currently are when one hot encoding.
# Keeping the order will be handy later on when visualising predictions.
renameDict = {"Small-flowered Cranesbill":"A",
"Fat Hen":"B",
"Shepherds Purse":"C",
"Common wheat":"D",
"Common Chickweed":"E",
"Charlock":"F",
"Cleavers":"G",
"Scentless Mayweed":"H",
"Sugar beet":"I",
"Maize":"J",
"Black-grass":"K",
"Loose Silky-bent":"L",}
# First one-hot encode the labels.
labels_ohe = labels.replace(renameDict)
labels_ohe = pd.get_dummies(labels_ohe)
labels_ohe.iloc[classPositions]
# The labels_ohe dataframe must be converted to an array.
labels_array = labels_ohe.to_numpy()
labels_array[classPositions]
# Notic that the classes are in the same order as the labels, so printing label from each class makes an identity matrix.
# This is purely to aid visualization of predictions later on.
# Check the shape of the X data.
images_gs.shape
# Split the data into training, validation sets.
X_train_gs, X_test_gs, y_train_gs, y_test_gs = train_test_split(images_gs, labels_array, test_size=0.3, random_state=9, stratify=labels_ohe)
# Splite the test data into validation and test datasets.
X_val_gs, X_test_gs, y_val_gs, y_test_gs = train_test_split(X_test_gs, y_test_gs, test_size=0.5, random_state=9, stratify=y_test_gs)
print("X_train shape:\t" + str(X_train_gs.shape))
print("y_train shape:\t" + str(y_train_gs.shape))
print("X_val shape:\t" + str(X_val_gs.shape))
print("y_val shape:\t" + str(y_val_gs.shape))
print("X_test shape:\t" + str(X_test_gs.shape))
print("y_test shape:\t" + str(y_test_gs.shape))
# The X data must be reshaped to (:, 128, 128, 1)
X_train_gs = X_train_gs.reshape(X_train_gs.shape[0], X_train_gs.shape[1], X_train_gs.shape[2], 1)
X_val_gs = X_val_gs.reshape(X_val_gs.shape[0], X_val_gs.shape[1], X_val_gs.shape[2], 1)
X_test_gs = X_test_gs.reshape(X_test_gs.shape[0], X_test_gs.shape[1], X_test_gs.shape[2], 1)
print("X_train shape:\t" + str(X_train_gs.shape))
print("X_val shape:\t" + str(X_val_gs.shape))
print("X_test shape:\t" + str(X_test_gs.shape))
# Confirm that the datatypes of the image arrays are floats.
print("X_train datatype:\t" + str(X_train_gs[0,0,0,0].dtype))
print("X_val datatype:\t\t" + str(X_val_gs[0,0,0,0].dtype))
print("X_test datatype:\t" + str(X_test_gs[0,0,0,0].dtype))
# Confirm that image array values are between 0 and 1.
print("X_train min and max:\t" + str(X_train_gs.min()) + " " + str(X_train_gs.max()))
print("X_val min and max:\t" + str(X_val_gs.min()) + " " + str(X_val_gs.max()))
print("X_test min and max:\t" + str(X_test_gs.min()) + " " + str(X_test_gs.max()))
# Build the model.
model_gs = Sequential()
model_gs.add(Conv2D(filters=32, kernel_size=3, activation="relu", input_shape=(128, 128, 1)))
model_gs.add(Conv2D(filters=32, kernel_size=3, activation="relu"))
model_gs.add(Flatten())
model_gs.add(Dense(128, activation="relu"))
model_gs.add(Dense(12, activation="softmax"))
# Compile the model.
model_gs.compile(loss="categorical_crossentropy", metrics=["accuracy"], optimizer="adam")
# View the model summary.
model_gs.summary()
# Train the model.
history_gs = model_gs.fit(X_train_gs, y_train_gs, batch_size=32, epochs=10, validation_data=(X_val_gs, y_val_gs), verbose=1)
# Plot training history for initial grayscale model.
def plotFitHistory(fitHistory):
f, ax = plt.subplots(1, 2, figsize=(12,5))
ax[0].plot(fitHistory.history['accuracy'], label='Train')
ax[0].plot(fitHistory.history['val_accuracy'], label='Validation')
ax[0].title.set_text("Accuracy")
ax[0].legend()
ax[1].plot(fitHistory.history['loss'], label='Train')
ax[1].plot(fitHistory.history['val_loss'], label='Validation')
ax[1].title.set_text("Loss")
ax[1].legend()
plotFitHistory(history_gs)
# Predict the classes based on the validation X data.
y_valPred_gs = model_gs.predict_classes(X_val_gs)
# Convert from one hot encoding to indices.
y_val_ind_gs = np.argmax(y_val_gs, axis=1)
y_val_ind_gs[0:5]
# Get the loss and accuracy based on the validation data.
loss_gs, accuracy_gs = model_gs.evaluate(X_val_gs, y_val_gs, verbose=0)
print("Validation Accuracy:\t{:.4f}".format(accuracy_gs))
print("Validation Loss:\t{:.4f}".format(loss_gs))
# Get the classification report based on the validation data.
print("Initial Grayscale Model Validation Classification Report")
print(classification_report(y_val_ind_gs, y_valPred_gs))
# Plot the validation confusion matrix for the initial grayscale model.
cm_gs=confusion_matrix(y_val_ind_gs, y_valPred_gs)
sns.heatmap(cm_gs, annot=True, fmt='g', xticklabels = [0,1,2,3,4,5,6,7,8,9,10,11] , yticklabels = [0,1,2,3,4,5,6,7,8,9,10,11] )
plt.title("Initial Grayscale Model Validation Confusion Matrix")
plt.ylabel('Actual')
plt.xlabel('Predicted')
# Y Data is already prepared.
# New X data must be made from the gaussian blurred colour images.
# Check the shape of the X data.
images_rgb.shape
# Split the data into training, validation sets.
X_train_rgb, X_test_rgb, y_train_rgb, y_test_rgb = train_test_split(images_rgb, labels_array, test_size=0.3, random_state=9, stratify=labels_ohe)
# Split the test data into validation and test datasets.
X_val_rgb, X_test_rgb, y_val_rgb, y_test_rgb = train_test_split(X_test_rgb, y_test_rgb, test_size=0.5, random_state=9, stratify=y_test_rgb)
print("X_train shape:\t" + str(X_train_rgb.shape))
print("y_train shape:\t" + str(y_train_rgb.shape))
print("X_val shape:\t" + str(X_val_rgb.shape))
print("y_val shape:\t" + str(y_val_rgb.shape))
print("X_test shape:\t" + str(X_test_rgb.shape))
print("y_test shape:\t" + str(y_test_rgb.shape))
# Confirm that image array values are between 0 and 1.
print("X_train min and max:\t" + str(X_train_rgb.min()) + " " + str(X_train_rgb.max()))
print("X_val min and max:\t" + str(X_val_rgb.min()) + " " + str(X_val_rgb.max()))
print("X_test min and max:\t" + str(X_test_rgb.min()) + " " + str(X_test_rgb.max()))
# Build the model.
model_rgb = Sequential()
model_rgb.add(Conv2D(filters=32, kernel_size=3, activation="relu", input_shape=(128, 128, 3), data_format='channels_last'))
model_rgb.add(Conv2D(filters=32, kernel_size=3, activation="relu"))
model_rgb.add(Flatten())
model_rgb.add(Dense(128, activation="relu"))
model_rgb.add(Dense(12, activation="softmax"))
# Compile the model.
model_rgb.compile(loss="categorical_crossentropy", metrics=["accuracy"], optimizer="adam")
# View the model summary.
model_rgb.summary()
# Train the model.
history_rgb = model_rgb.fit(X_train_rgb, y_train_rgb, batch_size=32, epochs=10, validation_data=(X_val_rgb, y_val_rgb), verbose=1)
# Plot training history for initial RGB model.
plotFitHistory(history_rgb)
# Make predictions based on the validation X data.
y_valPred_rgb = model_rgb.predict_classes(X_val_rgb)
# Convert from one hot encoding to indices.
y_val_ind_rgb = np.argmax(y_val_rgb, axis=1)
y_val_ind_rgb[0:5]
# Get the loss and the accuracy
loss_rgb, accuracy_rgb = model_rgb.evaluate(X_val_rgb, y_val_rgb, verbose=0)
print("Validation Accuracy:\t{:.4f}".format(accuracy_rgb))
print("Validation Loss:\t{:.4f}".format(loss_rgb))
print("Initial RGB Model Validation Classification Report")
print(classification_report(y_val_ind_rgb, y_valPred_rgb))
cm_rgb=confusion_matrix(y_val_ind_rgb, y_valPred_rgb)
sns.heatmap(cm_rgb, annot=True, fmt='g', xticklabels = [0,1,2,3,4,5,6,7,8,9,10,11] , yticklabels = [0,1,2,3,4,5,6,7,8,9,10,11] )
plt.title("Initial RGB Model Validation Confusion Matrix")
plt.ylabel('Actual')
plt.xlabel('Predicted')
# Continue with the RGB model, add some more layers to try and reduce overfitting and to get good results.
# Building the model was an iterative process, I ommitted showing each iteration and instead only show the model which I ended up with.
model_ref = Sequential()
model_ref.add(Conv2D(filters=32, kernel_size=3, padding='same', activation="relu", input_shape=(128, 128, 3), data_format='channels_last'))
model_ref.add(MaxPooling2D(pool_size=(2, 2)))
model_ref.add(Dropout(rate=0.3))
model_ref.add(Conv2D(filters=32, kernel_size=3, padding='same', activation="relu"))
model_ref.add(MaxPooling2D(pool_size=(2, 2)))
model_ref.add(Dropout(rate=0.2))
model_ref.add(Conv2D(filters=64, kernel_size=3, padding='same', activation="relu"))
model_ref.add(MaxPooling2D(pool_size=(2, 2)))
model_ref.add(Dropout(rate=0.1))
model_ref.add(Conv2D(filters=64, kernel_size=3, padding='same', activation="relu"))
model_ref.add(MaxPooling2D(pool_size=(2, 2)))
model_ref.add(Dropout(rate=0.1))
model_ref.add(GlobalMaxPooling2D())
model_ref.add(Flatten())
model_ref.add(Dense(512, activation="relu"))
model_ref.add(Dropout(rate=0.1))
model_ref.add(Dense(256, activation="relu"))
model_ref.add(Dropout(rate=0.1))
model_ref.add(Dense(128, activation="relu"))
model_ref.add(Dropout(rate=0.1))
model_ref.add(Dense(12, activation="softmax"))
# Compile the model.
model_ref.compile(loss="categorical_crossentropy", metrics=["accuracy"], optimizer="adam")
# View the model summary.
model_ref.summary()
# Train the model.
history_ref = model_ref.fit(X_train_rgb, y_train_rgb, batch_size=32, epochs=30, validation_data=(X_val_rgb, y_val_rgb), verbose=1)
# Plot training history for refined model.
plotFitHistory(history_ref)
# Make predictions based on the validation X data.
y_valPred_ref = model_ref.predict_classes(X_val_rgb)
# Convert from one hot encoding to indices.
y_val_ind_ref = np.argmax(y_val_rgb, axis=1)
# Get the training loss and the accuracy
trainLoss_ref, trainAccuracy_ref = model_ref.evaluate(X_train_rgb, y_train_rgb, verbose=0)
print("Training Accuracy:\t{:.4f}".format(trainAccuracy_ref))
print("Training Loss:\t\t{:.4f}".format(trainLoss_ref))
# Get the validation loss and the accuracy
validationLoss_ref, validationAccuracy_ref = model_ref.evaluate(X_val_rgb, y_val_rgb, verbose=0)
print("Validation Accuracy:\t{:.4f}".format(validationAccuracy_ref))
print("Validation Loss:\t{:.4f}".format(validationLoss_ref))
print("Refined Model Validation Classification Report")
print(classification_report(y_val_ind_ref, y_valPred_ref))
cm_rgb=confusion_matrix(y_val_ind_ref, y_valPred_ref)
sns.heatmap(cm_rgb, annot=True, fmt='g', xticklabels = [0,1,2,3,4,5,6,7,8,9,10,11] , yticklabels = [0,1,2,3,4,5,6,7,8,9,10,11] )
plt.title("Refined Model Validation Confusion Matrix")
plt.ylabel('Actual')
plt.xlabel('Predicted')
# There are numerous instances of class 10 incorrectly classified as class 11.
# There are also quite a few instances of class 11 incorrectly classified as class 10.
# If these can be corrected then that will increase the accuracy.
# Find these instances.
class10IncorrectAs11 = []
class11IncorrectAs10 = []
for i in range(len(y_valPred_ref)):
if((y_val_ind_ref[i] == 10) and (y_valPred_ref[i] == 11)):
class10IncorrectAs11.append(i)
elif ((y_val_ind_ref[i] == 11) and (y_valPred_ref[i] == 10)):
class11IncorrectAs10.append(i)
# print some pictures of class 11 to compare to the incorrect classifications of class 10.
print("Example images from ", classNames[11])
printImages(images_rgb, labelsList, np.array(range(classPositions[11],classPositions[11]+6, 1)), picSize=3)
# Print the incorrectly predicted images from class 10.
print("Images from ", classNames[10], " incorrectly classified as ", classNames[11])
printImages(X_val_rgb, y_val_ind_ref, class10IncorrectAs11, picSize=3)
# Print some pictures of class 10 to compare to the incorrect classifications of class 11.
print("Example images from ", classNames[10])
printImages(images_rgb, labelsList, np.array(range(classPositions[11],classPositions[11]+6, 1)), picSize=3)
# Print the incorrectly predicted images from class 11.
print("Images from ", classNames[11], " incorrectly classified as ", classNames[10])
printImages(X_val_rgb, y_val_ind_ref, class11IncorrectAs10, picSize=3)
# It is easy to see why these are two classes are easily confused with each other. Both are a thin blade lea plant.
# classifying these images into the correct class probably requires advance techniques or just better images.
# Make predictions based on the validation X data.
y_testPred_prob = model_ref.predict(X_test_rgb)
y_testPred_class = model_ref.predict_classes(X_test_rgb)
# Convert from one hot encoding to indices.
y_test_ind_ref = np.argmax(y_test_rgb, axis=1)
# Get the loss and the accuracy
testLoss_ref, testAccuracy_ref = model_ref.evaluate(X_test_rgb, y_test_rgb, verbose=0)
print("Test Accuracy:\t{:.4f}".format(testAccuracy_ref))
print("Test Loss:\t{:.4f}".format(testLoss_ref))
print("Refined Model Test Classification Report")
print(classification_report(y_test_ind_ref, y_testPred_class))
cmTest_rgb=confusion_matrix(y_test_ind_ref, y_testPred_class)
sns.heatmap(cmTest_rgb, annot=True, fmt='g', xticklabels = [0,1,2,3,4,5,6,7,8,9,10,11] , yticklabels = [0,1,2,3,4,5,6,7,8,9,10,11] )
plt.title("Refined Model Test Confusion Matrix")
plt.ylabel('Actual')
plt.xlabel('Predicted')
def visualizePrediction(index):
actualClass = y_test_ind_ref[index]
predictedClass = y_testPred_class[index]
print("Actual Class: ", actualClass)
print("Actual Name: "+classNames[actualClass])
plt.imshow(X_test_rgb[index])
plt.show()
print()
print("Predicted Class: ", predictedClass)
print("Predicted Name: "+classNames[predictedClass])
print("Softmax Probability Prediction: ")
print(y_testPred_prob[index])
if(actualClass != predictedClass):
print("Example of correct class")
plt.imshow(images_rgb[classPositions[predictedClass]])
plt.show()
print()
if(actualClass == predictedClass):
print("Correct Prediction")
else:
print("Incorrect Prediction")
print()
visualizePrediction(2)
visualizePrediction(3)
visualizePrediction(33)
visualizePrediction(36)
visualizePrediction(59)
# Visualize some incorrect predictions.
maxCount = 5
count = 0
for i in range(len(y_testPred_class)):
if(y_testPred_class[i] != y_test_ind_ref[i]):
visualizePrediction(i)
count += 1
if(count == maxCount):
break
trainDict = {'Loss': trainLoss_ref, 'Accuracy': trainAccuracy_ref}
trainSeries = pd.Series(name='Training', data=trainDict)
valDict = {'Loss': validationLoss_ref, 'Accuracy': validationAccuracy_ref}
valSeries = pd.Series(name='Validation', data=valDict)
testDict = {'Loss': testLoss_ref, 'Accuracy': testAccuracy_ref}
testSeries = pd.Series(name='Testing', data=testDict)
summary_df = pd.DataFrame(columns=['Accuracy', 'Loss'])
summary_df = summary_df.append(trainSeries)
summary_df = summary_df.append(valSeries)
summary_df = summary_df.append(testSeries)
summary_df